{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 649
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6902,
     "status": "ok",
     "timestamp": 1586610016414,
     "user": {
      "displayName": "Xue Jiang",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjfXAo9TmWFYS-GhDq8HOVk3N1FnJiyBPH8h9M=s64",
      "userId": "12544639825510830792"
     },
     "user_tz": -480
    },
    "id": "tbzwG1dEUdmc",
    "outputId": "5e1ee8a2-40aa-4b8c-8a31-f5f621357106"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pytorch-transformers\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a3/b7/d3d18008a67e0b968d1ab93ad444fc05699403fa662f634b2f2c318a508b/pytorch_transformers-1.2.0-py3-none-any.whl (176kB)\n",
      "\u001b[K     |████████████████████████████████| 184kB 4.9MB/s \n",
      "\u001b[?25hRequirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (1.12.38)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (2.21.0)\n",
      "Requirement already satisfied: torch>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (1.4.0)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (1.18.2)\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (4.38.0)\n",
      "Collecting sentencepiece\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
      "\u001b[K     |████████████████████████████████| 1.0MB 40.4MB/s \n",
      "\u001b[?25hRequirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (2019.12.20)\n",
      "Collecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/b4/7a41d630547a4afd58143597d5a49e07bfd4c42914d8335b2a5657efc14b/sacremoses-0.0.38.tar.gz (860kB)\n",
      "\u001b[K     |████████████████████████████████| 870kB 50.9MB/s \n",
      "\u001b[?25hRequirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-transformers) (0.9.5)\n",
      "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-transformers) (0.3.3)\n",
      "Requirement already satisfied: botocore<1.16.0,>=1.15.38 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-transformers) (1.15.38)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-transformers) (2020.4.5.1)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-transformers) (2.8)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-transformers) (3.0.4)\n",
      "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-transformers) (1.24.3)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->pytorch-transformers) (1.12.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->pytorch-transformers) (7.1.1)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->pytorch-transformers) (0.14.1)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.38->boto3->pytorch-transformers) (2.8.1)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.38->boto3->pytorch-transformers) (0.15.2)\n",
      "Building wheels for collected packages: sacremoses\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.38-cp36-none-any.whl size=884628 sha256=e023338d58a09b37160f8f6fe31dff687bba7707e9896e28fd784a61837dd004\n",
      "  Stored in directory: /root/.cache/pip/wheels/6d/ec/1a/21b8912e35e02741306f35f66c785f3afe94de754a0eaf1422\n",
      "Successfully built sacremoses\n",
      "Installing collected packages: sentencepiece, sacremoses, pytorch-transformers\n",
      "Successfully installed pytorch-transformers-1.2.0 sacremoses-0.0.38 sentencepiece-0.1.85\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 920,
     "status": "ok",
     "timestamp": 1586610680681,
     "user": {
      "displayName": "Xue Jiang",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjfXAo9TmWFYS-GhDq8HOVk3N1FnJiyBPH8h9M=s64",
      "userId": "12544639825510830792"
     },
     "user_tz": -480
    },
    "id": "Nr1uOsbGSx9Y",
    "outputId": "056ab7f4-923a-4664-c334-efd458ff38d6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['[CLS]', 'who', 'was', 'jim', 'henson', '?', '[SEP]', 'jim', 'henson', 'was', 'a', 'puppet', '##eer', '[SEP]']\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import BertTokenizer, BertModel, BertForMaskedLM\n",
    "\n",
    "# 加载预训练模型 tokenizer (vocabulary)\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "\n",
    "# 标记化输出\n",
    "text = \"[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]\"\n",
    "tokenized_text = tokenizer.tokenize(text)\n",
    "print(tokenized_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1077,
     "status": "ok",
     "timestamp": 1586610139378,
     "user": {
      "displayName": "Xue Jiang",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjfXAo9TmWFYS-GhDq8HOVk3N1FnJiyBPH8h9M=s64",
      "userId": "12544639825510830792"
     },
     "user_tz": -480
    },
    "id": "FBulQsxVU8To",
    "outputId": "5623fb5a-e378-43cd-8113-d584bd770657"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[101, 2040, 2001, 3958, 27227, 1029, 102, 3958, 103, 2001, 1037, 13997, 11510, 102]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# 掩码一个标记，我们将尝试用' BertForMaskedLM '预测回来\n",
    "masked_index = 8\n",
    "tokenized_text[masked_index] = '[MASK]'\n",
    "assert tokenized_text == ['[CLS]', 'who', 'was', 'jim', 'henson', '?', '[SEP]', 'jim', '[MASK]', 'was', 'a', 'puppet', '##eer', '[SEP]']\n",
    "\n",
    "# 将标记转换为词汇表索引\n",
    "indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)\n",
    "print(indexed_tokens)\n",
    "segments_ids = [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qTn9108dVUi3"
   },
   "outputs": [],
   "source": [
    "# 将输入转换为PyTorch张量\n",
    "tokens_tensor = torch.tensor([indexed_tokens])\n",
    "segments_tensors = torch.tensor([segments_ids])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 305
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 5180,
     "status": "ok",
     "timestamp": 1586610732414,
     "user": {
      "displayName": "Xue Jiang",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjfXAo9TmWFYS-GhDq8HOVk3N1FnJiyBPH8h9M=s64",
      "userId": "12544639825510830792"
     },
     "user_tz": -480
    },
    "id": "0HMsM1-SVpQr",
    "outputId": "d3786325-8f92-4376-c3e4-909de133e9f0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(tensor([[[ -7.8798,  -7.7874,  -7.7861,  ...,  -7.0438,  -6.7454,  -4.6013],\n",
      "         [-13.3633, -13.7694, -13.7819,  ..., -11.8128, -11.1635, -13.8906],\n",
      "         [-10.9775, -10.5383, -10.9659,  ..., -11.5549,  -8.0309,  -6.3979],\n",
      "         ...,\n",
      "         [ -5.2284,  -5.6572,  -5.3550,  ...,  -3.4507,  -3.8718,  -8.6904],\n",
      "         [ -8.5290,  -8.4146,  -9.0744,  ...,  -7.1710,  -6.9877,  -6.1301],\n",
      "         [-12.5968, -12.3769, -12.4222,  ..., -10.1020,  -9.8764,  -9.4495]]],\n",
      "       device='cuda:0'),)\n",
      "tensor([[[ -7.8798,  -7.7874,  -7.7861,  ...,  -7.0438,  -6.7454,  -4.6013],\n",
      "         [-13.3633, -13.7694, -13.7819,  ..., -11.8128, -11.1635, -13.8906],\n",
      "         [-10.9775, -10.5383, -10.9659,  ..., -11.5549,  -8.0309,  -6.3979],\n",
      "         ...,\n",
      "         [ -5.2284,  -5.6572,  -5.3550,  ...,  -3.4507,  -3.8718,  -8.6904],\n",
      "         [ -8.5290,  -8.4146,  -9.0744,  ...,  -7.1710,  -6.9877,  -6.1301],\n",
      "         [-12.5968, -12.3769, -12.4222,  ..., -10.1020,  -9.8764,  -9.4495]]],\n",
      "       device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "# 加载预训练模型 (weights)\n",
    "model = BertForMaskedLM.from_pretrained('bert-base-uncased')\n",
    "model.eval()\n",
    "\n",
    "# ：如果你有GPU，把所有东西都放在cuda上\n",
    "tokens_tensor = tokens_tensor.to('cuda')\n",
    "segments_tensors = segments_tensors.to('cuda')\n",
    "model.to('cuda')\n",
    "\n",
    "# 预测所有的tokens\n",
    "with torch.no_grad():\n",
    "    outputs = model(tokens_tensor, token_type_ids=segments_tensors)\n",
    "    predictions = outputs[0]\n",
    "    print(outputs)\n",
    "    print(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 976,
     "status": "ok",
     "timestamp": 1586610759708,
     "user": {
      "displayName": "Xue Jiang",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjfXAo9TmWFYS-GhDq8HOVk3N1FnJiyBPH8h9M=s64",
      "userId": "12544639825510830792"
     },
     "user_tz": -480
    },
    "id": "_rn-aBdIV2JR",
    "outputId": "26953408-af89-41ec-883d-8c4eede16c4a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted token is: henson\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# 证实我们能够预测到'henson'\n",
    "predicted_index = torch.argmax(predictions[0, masked_index]).item()\n",
    "predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0]\n",
    "assert predicted_token == 'henson'\n",
    "print('Predicted token is:',predicted_token)\n"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyP5iAY9ANbJ6XpeWrCo0WNa",
   "collapsed_sections": [],
   "name": "为BERT训练LML.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
